Merge pull request #1526 from QwenLM/chore/no-tiktoken

chore: remove tiktoken dependency and use API-reported token counts

Author: tanzhenxin

docs/developers/development/npm.md

@@ -202,7 +202,7 @@ This is the most critical stage where files are moved and transformed into their
       - Copies README.md and LICENSE to dist/
       - Copies locales folder for internationalization
       - Creates a clean package.json for distribution with only necessary dependencies
-      - Includes runtime dependencies like tiktoken
+      - Keeps distribution dependencies minimal (no bundled runtime deps)
       - Maintains optional dependencies for node-pty
 
 2.  The JavaScript Bundle is Created:

esbuild.config.js

@@ -33,7 +33,6 @@ const external = [
   '@lydell/node-pty-linux-x64',
   '@lydell/node-pty-win32-arm64',
   '@lydell/node-pty-win32-x64',
-  'tiktoken',
 ];
 
 esbuild

package-lock.json

@@ -38,14 +38,15 @@
   "dependencies": {
     "@google/genai": "1.30.0",
     "@iarna/toml": "^2.2.5",
-    "@qwen-code/qwen-code-core": "file:../core",
     "@modelcontextprotocol/sdk": "^1.25.1",
+    "@qwen-code/qwen-code-core": "file:../core",
     "@types/update-notifier": "^6.0.8",
     "ansi-regex": "^6.2.2",
     "command-exists": "^1.2.9",
     "comment-json": "^4.2.5",
     "diff": "^7.0.0",
     "dotenv": "^17.1.0",
+    "extract-zip": "^2.0.1",
     "fzf": "^0.5.2",
     "glob": "^10.5.0",
     "highlight.js": "^11.11.1",
@@ -65,7 +66,6 @@
     "strip-json-comments": "^3.1.1",
     "tar": "^7.5.2",
     "undici": "^6.22.0",
-    "extract-zip": "^2.0.1",
     "update-notifier": "^7.3.1",
     "wrap-ansi": "9.0.2",
     "yargs": "^17.7.2",
@@ -74,6 +74,7 @@
   "devDependencies": {
     "@babel/runtime": "^7.27.6",
     "@google/gemini-cli-test-utils": "file:../test-utils",
+    "@qwen-code/qwen-code-test-utils": "file:../test-utils",
     "@testing-library/react": "^16.3.0",
     "@types/archiver": "^6.0.3",
     "@types/command-exists": "^1.2.3",
@@ -92,8 +93,7 @@
     "pretty-format": "^30.0.2",
     "react-dom": "^19.1.0",
     "typescript": "^5.3.3",
-    "vitest": "^3.1.1",
-    "@qwen-code/qwen-code-test-utils": "file:../test-utils"
+    "vitest": "^3.1.1"
   },
   "engines": {
     "node": ">=20"

packages/cli/package.json

@@ -63,7 +63,6 @@
     "shell-quote": "^1.8.3",
     "simple-git": "^3.28.0",
     "strip-ansi": "^7.1.0",
-    "tiktoken": "^1.0.21",
     "undici": "^6.22.0",
     "uuid": "^9.0.1",
     "ws": "^8.18.0"

packages/core/package.json

@@ -19,9 +19,7 @@ const mockTokenizer = {
 };
 
 vi.mock('../../utils/request-tokenizer/index.js', () => ({
-  getDefaultTokenizer: vi.fn(() => mockTokenizer),
-  DefaultRequestTokenizer: vi.fn(() => mockTokenizer),
-  disposeDefaultTokenizer: vi.fn(),
+  RequestTokenEstimator: vi.fn(() => mockTokenizer),
 }));
 
 type AnthropicCreateArgs = [unknown, { signal?: AbortSignal }?];
@@ -352,9 +350,7 @@ describe('AnthropicContentGenerator', () => {
       };
 
       const result = await generator.countTokens(request);
-      expect(mockTokenizer.calculateTokens).toHaveBeenCalledWith(request, {
-        textEncoding: 'cl100k_base',
-      });
+      expect(mockTokenizer.calculateTokens).toHaveBeenCalledWith(request);
       expect(result.totalTokens).toBe(50);
     });
 

packages/core/src/core/anthropicContentGenerator/anthropicContentGenerator.test.ts

@@ -25,7 +25,7 @@ type MessageCreateParamsNonStreaming =
   Anthropic.MessageCreateParamsNonStreaming;
 type MessageCreateParamsStreaming = Anthropic.MessageCreateParamsStreaming;
 type RawMessageStreamEvent = Anthropic.RawMessageStreamEvent;
-import { getDefaultTokenizer } from '../../utils/request-tokenizer/index.js';
+import { RequestTokenEstimator } from '../../utils/request-tokenizer/index.js';
 import { safeJsonParse } from '../../utils/safeJsonParse.js';
 import { AnthropicContentConverter } from './converter.js';
 
@@ -105,10 +105,8 @@ export class AnthropicContentGenerator implements ContentGenerator {
     request: CountTokensParameters,
   ): Promise<CountTokensResponse> {
     try {
-      const tokenizer = getDefaultTokenizer();
-      const result = await tokenizer.calculateTokens(request, {
-        textEncoding: 'cl100k_base',
-      });
+      const estimator = new RequestTokenEstimator();
+      const result = await estimator.calculateTokens(request);
 
       return {
         totalTokens: result.totalTokens,