factorize code

Author: severo

src/lib/utils.ts

@@ -1,6 +1,7 @@
 import { stringify } from 'hightable'
-import { AsyncBuffer, asyncBufferFromUrl, cachedAsyncBuffer } from 'hyparquet'
+import { AsyncBuffer } from 'hyparquet'
 import { AsyncBufferFrom } from './workers/types.js'
+import { fromToAsyncBuffer } from './workers/utils.js'
 
 /**
  * Helper function to join class names
@@ -13,17 +14,7 @@ export function cn(...names: (string | undefined | false)[]): string {
  * Convert AsyncBufferFromUrl to AsyncBuffer.
  */
 export function asyncBufferFrom(from: AsyncBufferFrom): Promise<AsyncBuffer> {
-  if ('url' in from) {
-    // Cached asyncBuffer for urls only
-    const key = JSON.stringify(from)
-    const cached = cache.get(key)
-    if (cached) return cached
-    const asyncBuffer = asyncBufferFromUrl(from).then(cachedAsyncBuffer)
-    cache.set(key, asyncBuffer)
-    return asyncBuffer
-  } else {
-    return from.file.arrayBuffer()
-  }
+  return fromToAsyncBuffer(from, cache)
 }
 const cache = new Map<string, Promise<AsyncBuffer>>()
 // TODO(SL): do we really want a singleton?

src/lib/workers/parquetWorker.ts

@@ -1,24 +1,11 @@
 import type { ColumnData } from 'hyparquet'
-import { AsyncBuffer, asyncBufferFromUrl, cachedAsyncBuffer, parquetRead } from 'hyparquet'
+import { AsyncBuffer, parquetRead } from 'hyparquet'
 import { compressors } from 'hyparquet-compressors'
-import type { AsyncBufferFrom, ChunkMessage, ClientMessage, ErrorMessage, ResultMessage } from './types.js'
+import type { ChunkMessage, ClientMessage, ErrorMessage, ResultMessage } from './types.js'
+import { fromToAsyncBuffer } from './utils.js'
 
 const cache = new Map<string, Promise<AsyncBuffer>>()
 
-export function asyncBufferFrom(from: AsyncBufferFrom): Promise<AsyncBuffer> {
-  if ('url' in from) {
-    // Cached asyncBuffer for urls only
-    const key = JSON.stringify(from)
-    const cached = cache.get(key)
-    if (cached) return cached
-    const asyncBuffer = asyncBufferFromUrl(from).then(cachedAsyncBuffer)
-    cache.set(key, asyncBuffer)
-    return asyncBuffer
-  } else {
-    return from.file.arrayBuffer()
-  }
-}
-
 function postChunkMessage ({ chunk, queryId }: ChunkMessage) {
   self.postMessage({ chunk, queryId })
 }
@@ -31,7 +18,7 @@ function postErrorMessage ({ error, queryId }: ErrorMessage) {
 
 self.onmessage = async ({ data }: { data: ClientMessage }) => {
   const { rowStart, rowEnd, columns, metadata, from, queryId } = data
-  const file = await asyncBufferFrom(from)
+  const file = await fromToAsyncBuffer(from, cache)
   try {
     await parquetRead({ metadata, file, rowStart, rowEnd, columns, compressors, onChunk })
     postResultMessage({ queryId })

src/lib/workers/utils.ts

@@ -0,0 +1,20 @@
+import { AsyncBuffer, asyncBufferFromUrl, cachedAsyncBuffer } from 'hyparquet'
+import { AsyncBufferFrom } from './types.js'
+
+export function fromToAsyncBuffer(from: AsyncBufferFrom, cache?: Map<string, Promise<AsyncBuffer>>): Promise<AsyncBuffer> {
+  if ('url' in from) {
+    // Cached asyncBuffer for urls only
+    const key = JSON.stringify(from)
+    if (cache) {
+      const cached = cache.get(key)
+      if (cached) return cached
+    }
+    const asyncBuffer = asyncBufferFromUrl(from).then(cachedAsyncBuffer)
+    if (cache) {
+      cache.set(key, asyncBuffer)
+    }
+    return asyncBuffer
+  } else {
+    return from.file.arrayBuffer()
+  }
+}