Wire up tokenizer worker pool via IPC

Added IPC integration to make the worker pool functional:
- Added TOKENS_COUNT_BULK IPC channel constant
- Updated IPCApi type with tokens.countBulk method
- Added tokens API to preload.ts
- Registered token handlers in ipcMain.ts
- Added worker pool cleanup on app quit

The worker pool is now fully wired up and will be called via IPC.
Dynamic imports ensure the worker is only loaded when actually used.

Author: ammar-agent

src/constants/ipc-constants.ts

@@ -38,6 +38,9 @@ export const IPC_CHANNELS = {
   // Window channels
   WINDOW_SET_TITLE: "window:setTitle",
 
+  // Token channels
+  TOKENS_COUNT_BULK: "tokens:countBulk",
+
   // Dynamic channel prefixes
   WORKSPACE_CHAT_PREFIX: "workspace:chat:",
   WORKSPACE_METADATA: "workspace:metadata",

src/main.ts

@@ -436,6 +436,21 @@ if (gotTheLock) {
     }
   });
 
+  // Cleanup worker threads on quit
+  app.on("will-quit", () => {
+    console.log("App will quit - cleaning up worker threads");
+    void (async () => {
+      try {
+        // Dynamic import is acceptable here - only loaded if worker was used
+        /* eslint-disable-next-line no-restricted-syntax */
+        const { tokenizerWorkerPool } = await import("@/services/tokenizerWorkerPool");
+        tokenizerWorkerPool.terminate();
+      } catch (error) {
+        console.error("Error terminating worker pool:", error);
+      }
+    })();
+  });
+
   app.on("activate", () => {
     // Only create window if app is ready and no window exists
     // This prevents "Cannot create BrowserWindow before app is ready" error

src/preload.ts

@@ -110,6 +110,10 @@ const api: IPCApi = {
   window: {
     setTitle: (title: string) => ipcRenderer.invoke(IPC_CHANNELS.WINDOW_SET_TITLE, title),
   },
+  tokens: {
+    countBulk: (model: string, texts: string[]) =>
+      ipcRenderer.invoke(IPC_CHANNELS.TOKENS_COUNT_BULK, model, texts),
+  },
 };
 
 // Expose the API along with platform/versions

src/services/ipcMain.ts

@@ -140,6 +140,7 @@ export class IpcMain {
 
     this.registerDialogHandlers(ipcMain);
     this.registerWindowHandlers(ipcMain);
+    this.registerTokenHandlers(ipcMain);
     this.registerWorkspaceHandlers(ipcMain);
     this.registerProviderHandlers(ipcMain);
     this.registerProjectHandlers(ipcMain);
@@ -174,6 +175,24 @@ export class IpcMain {
     });
   }
 
+  private registerTokenHandlers(ipcMain: ElectronIpcMain): void {
+    ipcMain.handle(
+      IPC_CHANNELS.TOKENS_COUNT_BULK,
+      async (_event, model: string, texts: string[]) => {
+        try {
+          // Offload to worker thread - keeps main process responsive
+          // Dynamic import is acceptable here - worker pool is lazy-loaded on first use
+          /* eslint-disable-next-line no-restricted-syntax */
+          const { tokenizerWorkerPool } = await import("@/services/tokenizerWorkerPool");
+          return await tokenizerWorkerPool.countTokens(model, texts);
+        } catch (error) {
+          log.error(`Failed to count tokens for model ${model}:`, error);
+          return null; // Tokenizer not loaded or error occurred
+        }
+      }
+    );
+  }
+
   private registerWorkspaceHandlers(ipcMain: ElectronIpcMain): void {
     ipcMain.handle(
       IPC_CHANNELS.WORKSPACE_CREATE,

src/types/ipc.ts

@@ -230,4 +230,7 @@ export interface IPCApi {
   window: {
     setTitle(title: string): Promise<void>;
   };
+  tokens: {
+    countBulk(model: string, texts: string[]): Promise<number[] | null>;
+  };
 }