Use asyncBufferFromUrl from hyparquet

Author: platypii

package.json

@@ -28,11 +28,11 @@
     "hyparquet-compressors": "0.1.4"
   },
   "devDependencies": {
-    "@rollup/plugin-commonjs": "28.0.0",
+    "@rollup/plugin-commonjs": "28.0.1",
     "@rollup/plugin-node-resolve": "15.3.0",
     "@rollup/plugin-replace": "6.0.1",
     "@rollup/plugin-terser": "0.4.4",
-    "@rollup/plugin-typescript": "12.1.0",
+    "@rollup/plugin-typescript": "12.1.1",
     "@testing-library/react": "16.0.1",
     "@types/node": "22.7.5",
     "@types/react": "18.3.11",
@@ -46,7 +46,7 @@
     "react-dom": "18.3.1",
     "rollup": "4.24.0",
     "rollup-plugin-postcss": "4.0.2",
-    "tslib": "2.7.0",
+    "tslib": "2.8.0",
     "typescript": "5.6.3",
     "typescript-eslint": "8.9.0",
     "vitest": "2.1.3"

public/build/app.min.js.map

@@ -1,6 +1,5 @@
 import type { DataFrame } from 'hightable'
-import { AsyncBuffer, FileMetaData, parquetSchema } from 'hyparquet'
-import { readableStreamToArrayBuffer } from './streamConverters.js'
+import { FileMetaData, parquetSchema } from 'hyparquet'
 import { AsyncBufferFrom, parquetQueryWorker } from './workers/parquetWorkerClient.js'
 
 /**
@@ -17,40 +16,3 @@ export function parquetDataFrame(from: AsyncBufferFrom, metadata: FileMetaData):
     sortable: true,
   }
 }
-
-export async function asyncBufferFrom(url: string): Promise<AsyncBuffer> {
-  // get byteLength with head
-  const res = await fetch(url, { method: 'HEAD' })
-  if (!res.ok) throw new Error(`Failed to fetch parquet file: ${res.statusText}`)
-  const contentLength = res.headers.get('Content-Length')
-  if (!contentLength) throw new Error('Content-Length header missing')
-  const byteLength = Number(contentLength)
-
-  // slice with range requests
-  return {
-    byteLength,
-    slice: async (start: number, end?: number) => {
-      const headers = new Headers({ Range: rangeString(start, end) })
-      const res = await fetch(url, { headers })
-      if (!res.ok || !res.body) throw new Error(`Failed to fetch parquet file: ${res.statusText}`)
-      return readableStreamToArrayBuffer(res.body)
-    },
-  }
-}
-
-
-/**
- * Convert a start and end byte offset into a range string.
- * If start is negative, end must be undefined.
- */
-export function rangeString(start: number, end?: number): string {
-  if (start < 0) {
-    if (end !== undefined) throw new Error(`invalid suffix range [${start}, ${end}]`)
-    return `bytes=${start}`
-  } else if (end !== undefined) {
-    if (start >= end) throw new Error(`invalid empty range [${start}, ${end}]`)
-    return `bytes=${start}-${end - 1}`
-  } else {
-    return `bytes=${start}-`
-  }
-}

public/build/worker.min.js

@@ -3,7 +3,7 @@ import { compressors } from 'hyparquet-compressors'
 
 self.onmessage = async ({ data }) => {
   const { metadata, asyncBuffer, rowStart, rowEnd, orderBy } = data
-  const file = await asyncBufferFromUrl(asyncBuffer.url)
+  const file = await asyncBufferFromUrl(asyncBuffer.url, asyncBuffer.byteLength)
   try {
     const result = await parquetQuery({
       metadata, file, rowStart, rowEnd, orderBy, compressors,