fix: allow tokenizer fallback when Piscina unavailable

Detect Bun runtime and bypass Piscina so token counting works in tests.
Assert encoding responses and reuse worker logic when pooling is off.
Disable Git signing in tests and align fixtures with current adapters.

Change-Id: I12fc5053de8f2fb906a99e14b97689a6f74d0d7f
Signed-off-by: Thomas Kosiewski <[email protected]>

Author: ThomasK33

src/git.test.ts

@@ -21,6 +21,7 @@ describe("createWorktree", () => {
     await execAsync(`git init`, { cwd: tempGitRepo });
     await execAsync(`git config user.email "[email protected]"`, { cwd: tempGitRepo });
     await execAsync(`git config user.name "Test User"`, { cwd: tempGitRepo });
+    await execAsync(`git config commit.gpgsign false`, { cwd: tempGitRepo });
     await execAsync(`echo "test" > README.md`, { cwd: tempGitRepo });
     await execAsync(`git add .`, { cwd: tempGitRepo });
     await execAsync(`git commit -m "Initial commit"`, { cwd: tempGitRepo });

src/runtime/sshConnectionPool.test.ts

@@ -1,4 +1,5 @@
 import * as os from "os";
+import * as path from "path";
 import { getControlPath } from "./sshConnectionPool";
 import type { SSHRuntimeConfig } from "./SSHRuntime";
 
@@ -127,6 +128,9 @@ describe("username isolation", () => {
 
     // The path should be deterministic for this user
     expect(controlPath).toBe(getControlPath(config));
-    expect(controlPath).toMatch(/^\/tmp\/cmux-ssh-[a-f0-9]{12}$/);
+
+    const expectedPrefix = path.join(os.tmpdir(), "cmux-ssh-");
+    expect(controlPath.startsWith(expectedPrefix)).toBe(true);
+    expect(controlPath).toMatch(/cmux-ssh-[a-f0-9]{12}$/);
   });
 });

src/services/ipcMain.ts

@@ -1,5 +1,4 @@
 import assert from "@/utils/assert";
-import crypto from "crypto";
 import type { BrowserWindow, IpcMain as ElectronIpcMain } from "electron";
 import { spawn, spawnSync } from "child_process";
 import * as fs from "fs";
@@ -29,7 +28,6 @@ import { BashExecutionService } from "@/services/bashExecutionService";
 import { InitStateManager } from "@/services/initStateManager";
 import { createRuntime } from "@/runtime/runtimeFactory";
 import type { RuntimeConfig } from "@/types/runtime";
-import { uniqueSuffix } from "@/utils/hasher";
 /**
  * IpcMain - Manages all IPC handlers and service coordination
  *

src/services/streamManager.test.ts

@@ -379,7 +379,7 @@ describe("StreamManager - Unavailable Tool Handling", () => {
       messageId: "test-message-1",
       token: "test-token",
       startTime: Date.now(),
-      model: "test-model",
+      model: "anthropic:claude-sonnet-4-5",
       historySequence: 1,
       parts: [],
       lastPartialWriteTime: 0,

src/stores/WorkspaceConsumerManager.ts

@@ -250,8 +250,8 @@ export class WorkspaceConsumerManager {
       typeof queueMicrotask === "function"
         ? queueMicrotask
         : (callback: () => void) => {
-          void Promise.resolve().then(callback);
-        };
+            void Promise.resolve().then(callback);
+          };
 
     schedule(() => {
       this.pendingNotifications.delete(workspaceId);

src/utils/main/tokenizer.ts

@@ -27,7 +27,7 @@ const DEFAULT_WARM_MODELS = [
 ] as const;
 
 const requireForResolve = createRequire(__filename);
-const workerPath = requireForResolve.resolve("./tokenizer-pool.worker");
+const workerPath = requireForResolve.resolve("./tokenizer.worker");
 const tokenizerPool = new Piscina({
   filename: workerPath,
   idleTimeout: Infinity,

src/utils/main/tokenizer.worker.ts

@@ -1,10 +1,7 @@
 import assert from "@/utils/assert";
 import { Tokenizer, models } from "ai-tokenizer";
 import type { ModelName } from "ai-tokenizer";
-// import * as claude from "ai-tokenizer/encoding/claude";
-// import * as o200k_base from "ai-tokenizer/encoding/o200k_base";
-//
-// import * as encoding from "ai-tokenizer/encoding";
+import * as encoding from "ai-tokenizer/encoding";
 
 export interface CountTokensInput {
   modelName: ModelName;
@@ -13,43 +10,25 @@ export interface CountTokensInput {
 
 const tokenizerCache = new Map<ModelName, Tokenizer>();
 
-type importedClaude = typeof import("ai-tokenizer/encoding/claude");
-type importedO200k_base = typeof import("ai-tokenizer/encoding/o200k_base");
-let claude: importedClaude | null = null;
-let o200k_base: importedO200k_base | null = null;
-
-async function jitLoadEncoding(encoding: string): Promise<importedClaude | importedO200k_base> {
-  if (encoding == "claude") {
-    claude ??= await import("ai-tokenizer/encoding/claude");
-    return claude;
-  }
-
-  o200k_base ??= await import("ai-tokenizer/encoding/o200k_base");
-  return o200k_base;
-}
-
-async function getTokenizer(modelName: ModelName): Promise<Tokenizer> {
+function getTokenizer(modelName: ModelName): Tokenizer {
   const cached = tokenizerCache.get(modelName);
   if (cached) {
     return cached;
   }
 
   const model = models[modelName];
   assert(model, `Unknown tokenizer model '${modelName}'`);
-  assert(
-    model.encoding == "claude" || model.encoding == "o200k_base",
-    `Unknown encoding '${model.encoding}' for '${modelName}'`
-  );
 
-  const encodingModule = await jitLoadEncoding(model.encoding);
+  const encodingModule = encoding[model.encoding];
+  assert(encodingModule, `Unknown tokenizer encoding '${model.encoding}'`);
 
   const tokenizer = new Tokenizer(encodingModule);
   tokenizerCache.set(modelName, tokenizer);
   return tokenizer;
 }
 
-export async function countTokens({ modelName, input }: CountTokensInput): Promise<number> {
-  const tokenizer = await getTokenizer(modelName);
+export function countTokens({ modelName, input }: CountTokensInput): number {
+  const tokenizer = getTokenizer(modelName);
   const count = tokenizer.count(input);
   return count;
 }

src/utils/tokens/tokenStatsCalculator.test.ts

@@ -343,7 +343,7 @@ describe("mergeResults", () => {
       { consumer: "Assistant", promise: Promise.resolve(200) },
     ];
     const results = [100, 200];
-    const toolDefinitions = new Map();
+    const toolDefinitions = new Map<string, number>();
     const systemMessageTokens = 0;
 
     const consumerMap = mergeResults(jobs, results, toolDefinitions, systemMessageTokens);
@@ -358,7 +358,7 @@ describe("mergeResults", () => {
       { consumer: "User", promise: Promise.resolve(50) },
     ];
     const results = [100, 50];
-    const toolDefinitions = new Map();
+    const toolDefinitions = new Map<string, number>();
     const systemMessageTokens = 0;
 
     const consumerMap = mergeResults(jobs, results, toolDefinitions, systemMessageTokens);
@@ -372,7 +372,7 @@ describe("mergeResults", () => {
       { consumer: "Read", promise: Promise.resolve(50) },
     ];
     const results = [100, 50];
-    const toolDefinitions = new Map([["Read", 25]]);
+    const toolDefinitions = new Map<string, number>([["Read", 25]]);
     const systemMessageTokens = 0;
 
     const consumerMap = mergeResults(jobs, results, toolDefinitions, systemMessageTokens);
@@ -384,7 +384,7 @@ describe("mergeResults", () => {
   test("adds system message tokens", () => {
     const jobs: TokenCountJob[] = [];
     const results: number[] = [];
-    const toolDefinitions = new Map();
+    const toolDefinitions = new Map<string, number>();
     const systemMessageTokens = 300;
 
     const consumerMap = mergeResults(jobs, results, toolDefinitions, systemMessageTokens);
@@ -398,7 +398,7 @@ describe("mergeResults", () => {
       { consumer: "Assistant", promise: Promise.resolve(100) },
     ];
     const results = [0, 100];
-    const toolDefinitions = new Map();
+    const toolDefinitions = new Map<string, number>();
     const systemMessageTokens = 0;
 
     const consumerMap = mergeResults(jobs, results, toolDefinitions, systemMessageTokens);