feat(api): implement worker-based token counting

Move token counting logic to a dedicated worker thread for improved performance.
Adds support for both text and image content blocks using cl100k_base encoding.
The worker is lazily initialized and reused across requests.

Key changes:

Create token-counter.worker.ts for threaded token counting
Update BaseProvider to use worker for countTokens method
Add proper cleanup and error handling
Support image token estimation

Author: samhvw8

knip.json

@@ -1,6 +1,6 @@
 {
 	"$schema": "https://unpkg.com/knip@latest/schema.json",
-	"entry": ["src/extension.ts", "src/activate/index.ts", "webview-ui/src/index.tsx", "src/api/providers/workers/token-counter.worker.ts"],
+	"entry": ["src/extension.ts", "src/activate/index.ts", "webview-ui/src/index.tsx"],
 	"project": ["src/**/*.ts", "webview-ui/src/**/*.{ts,tsx}"],
 	"ignore": [
 		"**/__mocks__/**",
@@ -19,6 +19,7 @@
 		"src/exports/**",
 		"src/schemas/ipc.ts",
 		"src/extension.ts",
+		"src/api/providers/workers/token-counter.worker.ts",
 		"scripts/**"
 	],
 	"workspaces": {

package-lock.json

@@ -450,6 +450,7 @@
 		"string-similarity": "^4.0.4",
 		"strip-ansi": "^7.1.0",
 		"strip-bom": "^5.0.0",
+		"tiktoken-node": "^0.0.7",
 		"tmp": "^0.2.3",
 		"tree-sitter-wasms": "^0.1.11",
 		"turndown": "^7.2.0",

package.json

@@ -11,25 +11,6 @@ const TOKEN_WORKER_PATH = "workers/token-counter.worker.js"
  * Base class for API providers that implements common functionality
  */
 export abstract class BaseProvider implements ApiHandler {
-	private isDestroyed: boolean = false
-
-	/**
-	 * Class destructor to ensure cleanup
-	 */
-	public async [Symbol.asyncDispose](): Promise<void> {
-		if (!this.isDestroyed) {
-			this.isDestroyed = true
-			await this.cleanup()
-		}
-	}
-
-	/**
-	 * Cleanup resources used by the provider
-	 * This method can be called explicitly or will be called automatically on destruction
-	 */
-	public async cleanup(): Promise<void> {
-		this.isDestroyed = true
-	}
 	abstract createMessage(systemPrompt: string, messages: Anthropic.Messages.MessageParam[]): ApiStream
 	abstract getModel(): { id: string; info: ModelInfo }
 

src/api/providers/base-provider.ts

@@ -1,6 +1,6 @@
 import { parentPort } from "worker_threads"
 import { Tiktoken } from "js-tiktoken/lite"
-import o200kBase from "js-tiktoken/ranks/o200k_base"
+import cl100kBase from "js-tiktoken/ranks/cl100k_base"
 import { Anthropic } from "@anthropic-ai/sdk"
 
 // Reuse the fudge factor used in the original code
@@ -22,7 +22,8 @@ parentPort?.on("message", async (content: Array<ContentBlock>) => {
 
 		// Lazily create encoder if it doesn't exist
 		if (!encoder) {
-			encoder = new Tiktoken(o200kBase)
+			// Use cl100k_base encoding which is used by Claude models
+			encoder = new Tiktoken(cl100kBase)
 		}
 
 		// Process each content block

src/api/providers/workers/token-counter.worker.ts

@@ -28,6 +28,28 @@ class MockApiHandler extends BaseProvider {
 			},
 		}
 	}
+
+	override async countTokens(content: Array<Anthropic.Messages.ContentBlockParam>): Promise<number> {
+		if (!content || content.length === 0) return 0
+
+		let totalTokens = 0
+		for (const block of content) {
+			if (block.type === "text") {
+				const text = block.text || ""
+				// Simple estimation: 1 token per 4 characters
+				totalTokens += Math.ceil(text.length / 4)
+			} else if (block.type === "image") {
+				const imageSource = block.source
+				if (imageSource && typeof imageSource === "object" && "data" in imageSource) {
+					const base64Data = imageSource.data as string
+					totalTokens += Math.ceil(Math.sqrt(base64Data.length)) * 1.5
+				} else {
+					totalTokens += 300 // Conservative estimate for unknown images
+				}
+			}
+		}
+		return totalTokens
+	}
 }
 
 // Create a singleton instance for tests