Add tokenizer comparison page and TOON format support

- Add docs/tokenizer.html for interactive format/token comparison - Add
tokenizer.js for token counting and visualization - Add toon.js and
toon-wrapper.js to support TOON format encoding/decoding - Update
index.html navigation to include Token Comparison - Update GitHub
workflow: rename to release.yml, add GitHub Release step, support
provenance, Node.js 20, and tag triggers

Author: SeanLuis

.github/workflows/npm-publish.yml .github/workflows/release.yml.github/workflows/npm-publish.yml renamed to .github/workflows/release.yml

@@ -1,20 +1,29 @@
-name: Publish to NPM
+name: Release
 
 on:
   release:
     types: [created]
+  push:
+    tags:
+      - "v*"
+
+permissions:
+  id-token: write
+  contents: write
 
 jobs:
-  build:
+  publish:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
 
       - name: Setup Node.js
         uses: actions/setup-node@v4
         with:
-          node-version: '18.x'
-          registry-url: 'https://registry.npmjs.org'
+          node-version: "20.x"
+          registry-url: "https://registry.npmjs.org"
 
       - name: Install dependencies
         run: |
@@ -31,9 +40,14 @@ jobs:
           cd nodejs-compressor
           npm test
 
-      - name: Publish to NPM
+      - name: Publish to NPM with provenance
         run: |
           cd nodejs-compressor
-          npm publish --access public
+          npm publish --access public --provenance
         env:
           NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+
+      - name: Create GitHub Release
+        run: npx changelogithub
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

docs/index.html

@@ -107,6 +107,13 @@ <h1 class="text-xl font-semibold mb-1">
                         <i data-lucide="bar-chart-2" class="w-4 h-4"></i>
                         Benchmarks
                     </a>
+                    <a
+                        href="tokenizer.html"
+                        class="inline-flex items-center gap-1.5 px-3 py-1.5 text-sm border rounded-md hover:bg-gray-50 bg-blue-50 border-blue-200"
+                    >
+                        <i data-lucide="hash" class="w-4 h-4"></i>
+                        Token Comparison
+                    </a>
                     <a
                         href="tools.html"
                         class="inline-flex items-center gap-1.5 px-3 py-1.5 text-sm border rounded-md hover:bg-gray-50"