From c4e130f4be52c8e6b2bcb793be6cadc0ea564c73 Mon Sep 17 00:00:00 2001 From: Daniel Sogl Date: Sat, 20 Sep 2025 17:55:02 +0200 Subject: [PATCH] refactor(runner): replace gpt-tokenizer with tiktoken --- package.json | 4 +-- pnpm-lock.yaml | 16 +++++----- runner/codegen/genkit/providers/open-ai.ts | 37 ++++++++++++++-------- 3 files changed, 34 insertions(+), 23 deletions(-) diff --git a/package.json b/package.json index e173e97..8813113 100644 --- a/package.json +++ b/package.json @@ -71,7 +71,6 @@ "file-type": "^21.0.0", "genkit": "^1.19.1", "genkitx-anthropic": "0.23.1", - "gpt-tokenizer": "^3.0.1", "handlebars": "^4.7.8", "limiter": "^3.0.0", "marked": "^16.1.1", @@ -79,9 +78,10 @@ "p-queue": "^8.1.0", "puppeteer": "^24.10.1", "sass": "^1.89.2", - "stylelint": "^16.21.1", "strict-csp": "^1.1.1", + "stylelint": "^16.21.1", "stylelint-config-recommended-scss": "^16.0.0", + "tiktoken": "^1.0.22", "tinyglobby": "^0.2.14", "tsx": "^4.20.3", "typescript": "^5.8.3", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b0cfb1d..14268dd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -68,9 +68,6 @@ importers: genkitx-anthropic: specifier: 0.23.1 version: 0.23.1(encoding@0.1.13)(genkit@1.19.1(@google-cloud/firestore@7.11.3(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.5.0(encoding@0.1.13))) - gpt-tokenizer: - specifier: ^3.0.1 - version: 3.0.1 handlebars: specifier: ^4.7.8 version: 4.7.8 @@ -101,6 +98,9 @@ importers: stylelint-config-recommended-scss: specifier: ^16.0.0 version: 16.0.1(postcss@8.5.6)(stylelint@16.24.0(typescript@5.9.2)) + tiktoken: + specifier: ^1.0.22 + version: 1.0.22 tinyglobby: specifier: ^0.2.14 version: 0.2.15 @@ -3817,9 +3817,6 @@ packages: resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} engines: {node: '>= 0.4'} - gpt-tokenizer@3.0.1: - resolution: {integrity: sha512-5jdaspBq/w4sWw322SvQj1Fku+CN4OAfYZeeEg8U7CWtxBz+zkxZ3h0YOHD43ee+nZYZ5Ud70HRN0ANcdIj4qg==} - graceful-fs@4.2.10: resolution: {integrity: sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==} @@ -5733,6 +5730,9 @@ packages: engines: {node: '>= 0.10.x'} hasBin: true + tiktoken@1.0.22: + resolution: {integrity: sha512-PKvy1rVF1RibfF3JlXBSP0Jrcw2uq3yXdgcEXtKTYn3QJ/cBRBHDnrJ5jHky+MENZ6DIPwNUGWpkVx+7joCpNA==} + tinycolor2@1.6.0: resolution: {integrity: sha512-XPaBkWQJdsf3pLKJV9p4qN/S+fm2Oj8AIPo1BTUhg5oxkvm9+SVEGFdhyOz7tTdUTfvxMiAs4sp6/eZO2Ew+pw==} @@ -10646,8 +10646,6 @@ snapshots: gopd@1.2.0: {} - gpt-tokenizer@3.0.1: {} - graceful-fs@4.2.10: {} graceful-fs@4.2.11: {} @@ -12940,6 +12938,8 @@ snapshots: error: 7.0.2 long: 2.4.0 + tiktoken@1.0.22: {} + tinycolor2@1.6.0: {} tinyglobby@0.2.14: diff --git a/runner/codegen/genkit/providers/open-ai.ts b/runner/codegen/genkit/providers/open-ai.ts index 9342327..c529b96 100644 --- a/runner/codegen/genkit/providers/open-ai.ts +++ b/runner/codegen/genkit/providers/open-ai.ts @@ -1,4 +1,3 @@ -import { ChatMessage } from 'gpt-tokenizer/GptEncoding'; import { GenkitPluginV2 } from 'genkit/plugin'; import { openAI } from '@genkit-ai/compat-oai/openai'; import { RateLimiter } from 'limiter'; @@ -7,8 +6,7 @@ import { PromptDataForCounting, RateLimitConfig, } from '../model-provider.js'; -import o3 from 'gpt-tokenizer/model/o3'; -import o4Mini from 'gpt-tokenizer/model/o4-mini'; +import { encoding_for_model } from 'tiktoken'; export class OpenAiModelProvider extends GenkitModelProvider { readonly apiKeyVariableName = 'OPENAI_API_KEY'; @@ -19,6 +17,21 @@ export class OpenAiModelProvider extends GenkitModelProvider { 'openai-gpt-5': () => openAI.model('gpt-5'), }; + private countTokensForModel( + modelName: Parameters[0], + prompt: PromptDataForCounting + ): number { + const encoding = encoding_for_model(modelName); + try { + const messages = this.genkitPromptToOpenAi(prompt); + const text = messages.map((m) => `${m.role}: ${m.content}`).join('\n'); + const tokens = encoding.encode(text); + return tokens.length; + } finally { + encoding.free(); + } + } + protected rateLimitConfig: Record = { // See: https://platform.openai.com/docs/models/o3 'openai/o3': { @@ -30,8 +43,7 @@ export class OpenAiModelProvider extends GenkitModelProvider { tokensPerInterval: 30_000 * 0.75, // *0.75 to be more resilient to token count deviations interval: 1000 * 60 * 1.5, // Refresh tokens after 1.5 minutes to be on the safe side. }), - countTokens: async (prompt) => - o3.countTokens(this.genkitPromptToOpenAi(prompt)), + countTokens: async (prompt) => this.countTokensForModel('gpt-4o', prompt), }, // See https://platform.openai.com/docs/models/o4-mini 'openai/o4-mini': { @@ -44,7 +56,7 @@ export class OpenAiModelProvider extends GenkitModelProvider { interval: 1000 * 60 * 1.5, // Refresh tokens after 1.5 minutes to be on the safe side. }), countTokens: async (prompt) => - o4Mini.countTokens(this.genkitPromptToOpenAi(prompt)), + this.countTokensForModel('gpt-4o-mini', prompt), }, // See: https://platform.openai.com/docs/models/gpt-5 'openai/gpt-5': { @@ -56,10 +68,7 @@ export class OpenAiModelProvider extends GenkitModelProvider { tokensPerInterval: 30_000 * 0.75, // *0.75 to be more resilient to token count deviations interval: 1000 * 60 * 1.5, // Refresh tokens after 1.5 minutes to be on the safe side. }), - // TODO: at the time of writing, the `gpt-tokenizer` doesn't support gpt-5. - // See https://github.com/niieani/gpt-tokenizer/issues/73 - countTokens: async (prompt) => - o3.countTokens(this.genkitPromptToOpenAi(prompt)), + countTokens: async (prompt) => this.countTokensForModel('gpt-5', prompt), }, }; @@ -72,8 +81,10 @@ export class OpenAiModelProvider extends GenkitModelProvider { return {}; } - private genkitPromptToOpenAi(prompt: PromptDataForCounting): ChatMessage[] { - const openAiPrompt: string | ChatMessage[] = []; + private genkitPromptToOpenAi( + prompt: PromptDataForCounting + ): Array<{ role: string; content: string }> { + const openAiPrompt: Array<{ role: string; content: string }> = []; for (const part of prompt.messages) { for (const c of part.content) { openAiPrompt.push({ @@ -82,6 +93,6 @@ export class OpenAiModelProvider extends GenkitModelProvider { }); } } - return [...openAiPrompt, { content: prompt.prompt }]; + return [...openAiPrompt, { role: 'user', content: prompt.prompt }]; } }