hyparam · severo · Nov 15, 2024 · Nov 13, 2024 · Nov 14, 2024 · Nov 14, 2024
diff --git a/.github/workflows/ci_apps_hyparquet_demo.yml b/.github/workflows/ci_apps_hyparquet_demo.yml
@@ -0,0 +1,32 @@
+name: apps/hyparquet-demo
+on:
+  push:
+    paths:
+      - 'apps/hyparquet-demo/**'
+      - '.github/workflows/ci_apps_hyparquet_demo.yml'
+
+defaults:
+  run:
+    working-directory: ./apps/hyparquet-demo
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: npm i
+      - run: npm run lint
+
+  typecheck:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: npm i
+      - run: tsc
+
+  buildcheck:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: npm i
+      - run: npm run build
diff --git a/README.md b/README.md
@@ -8,3 +8,4 @@ It contains the following package:
 It also contains the following applications:
 - [`hyperparam`](./apps/cli): a cli tool for viewing arbitrarily large datasets in the browser.
 - [`hightable-demo`](./apps/hightable-demo): an example project showing how to use [hightable](https://github.com/hyparam/hightable).
+- [`hyparquet-demo`](./apps/hyparquet-demo): an example project showing how to use [hyparquet](https://github.com/hyparam/hyparquet).
diff --git a/apps/hyparquet-demo/.gitignore b/apps/hyparquet-demo/.gitignore
@@ -0,0 +1,24 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?
diff --git a/apps/hyparquet-demo/README.md b/apps/hyparquet-demo/README.md
@@ -0,0 +1,18 @@
+# Hyparquet demo
+
+This is an example project showing how to use [hyparquet](https://github.com/hyparam/hyparquet).
+
+## Build
+
+```bash
+cd apps/hyparquet-demo
+npm i
+npm run build
+```
+
+The build artifacts will be stored in the `dist/` directory and can be served using any static server, eg. `http-server`:
+
+```bash
+npm i -g http-server
+http-server dist/
+```
diff --git a/apps/hyparquet-demo/eslint.config.js b/apps/hyparquet-demo/eslint.config.js
@@ -0,0 +1,43 @@
+import js from '@eslint/js'
+import react from 'eslint-plugin-react'
+import reactHooks from 'eslint-plugin-react-hooks'
+import reactRefresh from 'eslint-plugin-react-refresh'
+import globals from 'globals'
+import tseslint from 'typescript-eslint'
+import { sharedJsRules, sharedTsRules } from '../../shared.eslint.config.js'
+
+export default tseslint.config(
+  { ignores: ['dist'] },
+  {
+    extends: [js.configs.recommended, ...tseslint.configs.strictTypeChecked, ...tseslint.configs.stylisticTypeChecked],
+    // Set the react version
+    settings: { react: { version: '18.3' } },
+    files: ['src/**/*.{ts,tsx}'],
+    languageOptions: {
+      ecmaVersion: 2020,
+      globals: globals.browser,
+      parserOptions: {
+        project: './tsconfig.json',
+        tsconfigRootDir: import.meta.dirname,
+      },
+    },
+    plugins: {
+      react,
+      'react-hooks': reactHooks,
+      'react-refresh': reactRefresh,
+    },
+    rules: {
+      ...react.configs.recommended.rules,
+      ...react.configs['jsx-runtime'].rules,
+      ...reactHooks.configs.recommended.rules,
+      'react-refresh/only-export-components': [
+        'warn',
+        { allowConstantExport: true },
+      ],
+      ...js.configs.recommended.rules,
+      ...tseslint.configs.recommended.rules,
+      ...sharedJsRules,
+      ...sharedTsRules,
+    },
+  },
+)
diff --git a/apps/hyparquet-demo/index.html b/apps/hyparquet-demo/index.html
@@ -0,0 +1,27 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8">
+    <title>hyparquet parquet file parser demo</title>
+    <link rel="icon" href="favicon.png" />
+    <!-- <link rel="stylesheet" href="demo/demo.css"> -->
+    <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Mulish:wght@400;600&display=swap"/>
+    <meta name="description" content="Online demo of hyparquet: a parser for apache parquet files. Drag and drop parquet files to view parquet data.">
+    <meta name="author" content="Hyperparam">
+    <meta name="keywords" content="hyparquet, parquet, parquet file, parquet parser, parquet reader, parquet viewer, parquet data, apache parquet, hightable">
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+  </head>
+  <body>
+    <nav>
+      <a class="brand" href='https://hyparam.github.io/hyparquet/'>
+        hyparquet
+      </a>
+    </nav>
+    <main id="content">
+      <div id="app"></div>
+    </main>
+    <input id="file-input" type="file">
+
+    <script type="module" src="/src/main.tsx"></script>
+  </body>
+</html>
diff --git a/apps/hyparquet-demo/package.json b/apps/hyparquet-demo/package.json
@@ -0,0 +1,33 @@
+{
+  "name": "hyparquet-demo",
+  "private": true,
+  "version": "0.0.0",
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "tsc -b && vite build",
+    "lint": "eslint .",
+    "preview": "vite preview"
+  },
+  "dependencies": {
+    "hyparquet": "1.5.0",
+    "hyparquet-compressors": "0.1.4",
+    "hightable": "0.7.0",
+    "react": "^18.3.1",
+    "react-dom": "^18.3.1"
+  },
+  "devDependencies": {
+    "@eslint/js": "^9.13.0",
+    "@types/react": "^18.3.12",
+    "@types/react-dom": "^18.3.1",
+    "@vitejs/plugin-react": "^4.3.3",
+    "eslint": "^9.13.0",
+    "eslint-plugin-react": "^7.37.2",
+    "eslint-plugin-react-hooks": "^5.0.0",
+    "eslint-plugin-react-refresh": "^0.4.14",
+    "globals": "^15.11.0",
+    "typescript": "~5.6.2",
+    "typescript-eslint": "^8.11.0",
+    "vite": "^5.4.10"
+  }
+}
diff --git a/apps/hyparquet-demo/public/favicon.png b/apps/hyparquet-demo/public/favicon.png
diff --git a/apps/hyparquet-demo/src/App.tsx b/apps/hyparquet-demo/src/App.tsx
@@ -0,0 +1,85 @@
+import { ReactNode } from 'react'
+import Page, { PageProps } from './Page.js'
+import Welcome from './Welcome.js'
+
+import { DataFrame, rowCache } from 'hightable'
+import { FileMetaData, byteLengthFromUrl, parquetMetadataAsync, parquetSchema } from 'hyparquet'
+import { useCallback, useEffect, useState } from 'react'
+import Dropzone from './Dropzone.js'
+import Layout from './Layout.js'
+import { asyncBufferFrom } from './utils.js'
+import { parquetQueryWorker } from './workers/parquetWorkerClient.js'
+import { AsyncBufferFrom, Row } from './workers/types.js'
+
+export default function App(): ReactNode {
+  const params = new URLSearchParams(location.search)
+  const url = params.get('key') ?? undefined
+
+  const [error, setError] = useState<Error>()
+  const [pageProps, setPageProps] = useState<PageProps>()
+
+  const setUnknownError = useCallback((e: unknown) => {
+    setError(e instanceof Error ? e : new Error(String(e)))
+  }, [])
+
+  const onUrlDrop = useCallback(
+    (url: string) => {
+      // Add key=url to query string
+      const params = new URLSearchParams(location.search)
+      params.set('key', url)
+      history.pushState({}, '', `${location.pathname}?${params}`)
+      byteLengthFromUrl(url).then(byteLength => setAsyncBuffer(url, { url, byteLength })).catch(setUnknownError)
+    },
+    [setUnknownError],
+  )
+
+  useEffect(() => {
+    if (!pageProps && url) {
+      onUrlDrop(url)
+    }
+  }, [ url, pageProps, onUrlDrop])
+
+  function onFileDrop(file: File) {
+    // Clear query string
+    history.pushState({}, '', location.pathname)
+    setAsyncBuffer(file.name, { file, byteLength: file.size }).catch(setUnknownError)
+  }
+
+  async function setAsyncBuffer(name: string, from: AsyncBufferFrom) {
+    const asyncBuffer = await asyncBufferFrom(from)
+    const metadata = await parquetMetadataAsync(asyncBuffer)
+    const df = rowCache(parquetDataFrame(from, metadata))
+    setPageProps({ metadata, df, name, byteLength: from.byteLength, setError })
+  }
+
+  return <Layout error={error}>
+    <Dropzone
+      onError={(e) => { setError(e) }}
+      onFileDrop={onFileDrop}
+      onUrlDrop={onUrlDrop}>
+      {pageProps ? <Page {...pageProps} /> : <Welcome />}
+    </Dropzone>
+  </Layout>
+}
+
+/**
+ * Convert a parquet file into a dataframe.
+ */
+function parquetDataFrame(from: AsyncBufferFrom, metadata: FileMetaData): DataFrame {
+  const { children } = parquetSchema(metadata)
+  return {
+    header: children.map(child => child.element.name),
+    numRows: Number(metadata.num_rows),
+    /**
+     * @param {number} rowStart
+     * @param {number} rowEnd
+     * @param {string} orderBy
+     * @returns {Promise<any[][]>}
+     */
+    rows(rowStart: number, rowEnd: number, orderBy: string): Promise<Row[]> {
+      console.log(`reading rows ${rowStart}-${rowEnd}`, orderBy)
+      return parquetQueryWorker({ from, metadata, rowStart, rowEnd, orderBy })
+    },
+    sortable: true,
+  }
+}
diff --git a/apps/hyparquet-demo/src/Dropdown.tsx b/apps/hyparquet-demo/src/Dropdown.tsx
@@ -0,0 +1,72 @@
+import { ReactNode, useEffect, useRef, useState } from 'react'
+import { cn } from './utils.js'
+
+interface DropdownProps {
+  label?: string
+  className?: string
+  children: ReactNode
+}
+
+/**
+ * Dropdown menu component.
+ *
+ * @param {Object} props
+ * @param {string} props.label - button label
+ * @param {string} props.className - custom class name for the dropdown container
+ * @param {ReactNode} props.children - dropdown menu items
+ * @returns {ReactNode}
+ * @example
+ * <Dropdown label='Menu'>
+ *  <button>Item 1</button>
+ *  <button>Item 2</button>
+ * </Dropdown>
+ */
+export default function Dropdown({ label, className, children }: DropdownProps): ReactNode {
+  const [isOpen, setIsOpen] = useState(false)
+  const dropdownRef = useRef<HTMLDivElement>(null)
+  const menuRef = useRef<HTMLDivElement>(null)
+
+  function toggleDropdown() {
+    setIsOpen(!isOpen)
+  }
+
+  useEffect(() => {
+    function handleClickInside(event: MouseEvent) {
+      const target = event.target as Element
+      if (menuRef.current && menuRef.current.contains(target) && target.tagName !== 'INPUT') {
+        setIsOpen(false)
+      }
+    }
+    function handleClickOutside(event: MouseEvent) {
+      if (dropdownRef.current && !dropdownRef.current.contains(event.target as Node)) {
+        setIsOpen(false)
+      }
+    }
+    function handleEscape(event: KeyboardEvent) {
+      if (event.key === 'Escape') {
+        setIsOpen(false)
+      }
+    }
+    document.addEventListener('click', handleClickInside)
+    document.addEventListener('keydown', handleEscape)
+    document.addEventListener('mousedown', handleClickOutside)
+    return () => {
+      document.removeEventListener('click', handleClickInside)
+      document.removeEventListener('keydown', handleEscape)
+      document.removeEventListener('mousedown', handleClickOutside)
+    }
+  }, [])
+
+  return (
+    <div
+      className={cn('dropdown', className, isOpen && 'open')}
+      ref={dropdownRef}>
+      <button className='dropdown-button' onClick={toggleDropdown}>
+        {label}
+      </button>
+      <div className='dropdown-content' ref={menuRef}>
+        {children}
+      </div>
+    </div>
+  )
+}