feat: enable async token counting with piscina workers

Port tokenizer logic into a Piscina worker pool so token counting no
longer blocks the main thread and exposes async APIs across IPC.
Render live token estimates in chat input with Suspense fallback and
tighten defensive assertions while adapting tests to the new async flow.

Author: ThomasK33

.github/workflows/chromatic.yml

@@ -36,4 +36,3 @@ jobs:
           projectToken: ${{ secrets.CHROMATIC_PROJECT_TOKEN }}
           exitZeroOnChanges: true
           onlyChanged: true
-

.github/workflows/nightly-terminal-bench.yml

@@ -3,13 +3,13 @@ name: Nightly Terminal-Bench
 on:
   schedule:
     # Run full benchmark suite (~80 tasks) every night at midnight UTC
-    - cron: '0 0 * * *'
+    - cron: "0 0 * * *"
   workflow_dispatch:
     inputs:
       models:
         description: 'Models to test (comma-separated, or "all" for both)'
         required: false
-        default: 'all'
+        default: "all"
         type: string
 
 jobs:
@@ -41,9 +41,9 @@ jobs:
     uses: ./.github/workflows/terminal-bench.yml
     with:
       model_name: ${{ matrix.model }}
-      thinking_level: 'high'
-      dataset: 'terminal-bench-core==0.1.1'
-      concurrency: '4'
+      thinking_level: "high"
+      dataset: "terminal-bench-core==0.1.1"
+      concurrency: "4"
       livestream: true
     secrets:
       ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}

.github/workflows/publish-npm.yml

@@ -5,7 +5,7 @@ on:
     branches:
       - main
     tags:
-      - 'v*'
+      - "v*"
   workflow_dispatch:
 
 permissions:
@@ -24,12 +24,12 @@ jobs:
 
       - uses: ./.github/actions/setup-cmux
         with:
-          install-imagemagick: 'true'
+          install-imagemagick: "true"
 
       # Sets up .npmrc with the auth token
       - uses: actions/setup-node@v4
         with:
-          registry-url: 'https://registry.npmjs.org'
+          registry-url: "https://registry.npmjs.org"
 
       - run: sudo npm i -g npm@latest
 
@@ -38,10 +38,10 @@ jobs:
         run: |
           # Get base version from package.json
           BASE_VERSION=$(node -p "require('./package.json').version")
-          
+
           # Generate git describe version
           GIT_DESCRIBE=$(git describe --tags --always --dirty 2>/dev/null || echo "unknown")
-          
+
           if [[ $GITHUB_REF == refs/tags/* ]]; then
             # For tags, use the base version as-is (stable release)
             NPM_VERSION="${BASE_VERSION}"
@@ -56,13 +56,13 @@ jobs:
             NPM_TAG="next"
             echo "Publishing pre-release: ${NPM_VERSION}"
           fi
-          
+
           echo "version=${NPM_VERSION}" >> $GITHUB_OUTPUT
           echo "tag=${NPM_TAG}" >> $GITHUB_OUTPUT
-          
+
           # Update package.json with the new version
           node -e "const fs = require('fs'); const pkg = JSON.parse(fs.readFileSync('package.json')); pkg.version = '${NPM_VERSION}'; fs.writeFileSync('package.json', JSON.stringify(pkg, null, 2) + '\n');"
-          
+
           echo "Updated package.json to version ${NPM_VERSION}"
 
       - name: Generate version file
@@ -76,7 +76,7 @@ jobs:
         run: |
           PACKAGE_NAME=$(node -p "require('./package.json').name")
           VERSION="${{ steps.version.outputs.version }}"
-          
+
           if npm view "${PACKAGE_NAME}@${VERSION}" version &>/dev/null; then
             echo "exists=true" >> $GITHUB_OUTPUT
             echo "Version ${VERSION} already exists on npm"
@@ -95,7 +95,7 @@ jobs:
           PACKAGE_NAME=$(node -p "require('./package.json').name")
           VERSION="${{ steps.version.outputs.version }}"
           TAG="${{ steps.version.outputs.tag }}"
-          
+
           echo "Version ${VERSION} already published, updating dist-tag to ${TAG}"
           npm dist-tag add "${PACKAGE_NAME}@${VERSION}" "${TAG}"
 

.github/workflows/terminal-bench.yml

@@ -4,34 +4,34 @@ on:
   workflow_call:
     inputs:
       model_name:
-        description: 'Model to use (e.g., anthropic:claude-sonnet-4-5)'
+        description: "Model to use (e.g., anthropic:claude-sonnet-4-5)"
         required: false
         type: string
       thinking_level:
-        description: 'Thinking level (off, low, medium, high)'
+        description: "Thinking level (off, low, medium, high)"
         required: false
         type: string
       dataset:
-        description: 'Terminal-Bench dataset to use'
+        description: "Terminal-Bench dataset to use"
         required: false
         type: string
-        default: 'terminal-bench-core==0.1.1'
+        default: "terminal-bench-core==0.1.1"
       concurrency:
-        description: 'Number of concurrent tasks (--n-concurrent)'
+        description: "Number of concurrent tasks (--n-concurrent)"
         required: false
         type: string
-        default: '4'
+        default: "4"
       livestream:
-        description: 'Enable livestream mode'
+        description: "Enable livestream mode"
         required: false
         type: boolean
         default: true
       sample_size:
-        description: 'Number of random tasks to run (empty = all tasks)'
+        description: "Number of random tasks to run (empty = all tasks)"
         required: false
         type: string
       extra_args:
-        description: 'Additional arguments to pass to terminal-bench'
+        description: "Additional arguments to pass to terminal-bench"
         required: false
         type: string
     secrets:
@@ -42,34 +42,34 @@ on:
   workflow_dispatch:
     inputs:
       dataset:
-        description: 'Terminal-Bench dataset to use'
+        description: "Terminal-Bench dataset to use"
         required: false
-        default: 'terminal-bench-core==0.1.1'
+        default: "terminal-bench-core==0.1.1"
         type: string
       concurrency:
-        description: 'Number of concurrent tasks (--n-concurrent)'
+        description: "Number of concurrent tasks (--n-concurrent)"
         required: false
-        default: '4'
+        default: "4"
         type: string
       livestream:
-        description: 'Enable livestream mode'
+        description: "Enable livestream mode"
         required: false
         default: true
         type: boolean
       sample_size:
-        description: 'Number of random tasks to run (empty = all tasks)'
+        description: "Number of random tasks to run (empty = all tasks)"
         required: false
         type: string
       model_name:
-        description: 'Model to use (e.g., anthropic:claude-sonnet-4-5, openai:gpt-5-codex)'
+        description: "Model to use (e.g., anthropic:claude-sonnet-4-5, openai:gpt-5-codex)"
         required: false
         type: string
       thinking_level:
-        description: 'Thinking level (off, low, medium, high)'
+        description: "Thinking level (off, low, medium, high)"
         required: false
         type: string
       extra_args:
-        description: 'Additional arguments to pass to terminal-bench'
+        description: "Additional arguments to pass to terminal-bench"
         required: false
         type: string
 
@@ -148,4 +148,3 @@ jobs:
             runs/
           if-no-files-found: warn
           retention-days: 30
-

.storybook/main.ts

@@ -4,11 +4,7 @@ import path from "path";
 
 const config: StorybookConfig = {
   stories: ["../src/**/*.stories.@(ts|tsx)"],
-  addons: [
-    "@storybook/addon-links",
-    "@storybook/addon-docs",
-    "@storybook/addon-interactions",
-  ],
+  addons: ["@storybook/addon-links", "@storybook/addon-docs", "@storybook/addon-interactions"],
   framework: {
     name: "@storybook/react-vite",
     options: {},

.storybook/mocks/version.ts

@@ -6,4 +6,3 @@ export const VERSION = {
   git_describe: "v1.0.0",
   buildTime: "2024-01-24T17:41:00Z", // 9:41 AM PST
 };
-